package com.example.origin.spider.webmagic;

import cn.hutool.extra.spring.SpringUtil;
import com.example.origin.spider.properties.ProcessorProperties;
import com.example.origin.spider.properties.WebMagicProperties;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Collections;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

@Slf4j
public abstract class AbstractProcessor implements PageProcessor, Pipeline {


    protected void process(Page page, ProcessorProperties properties) {
    }

    @Override
    public void process(ResultItems resultItems, Task task) {
        ProcessorProperties properties = getProcessProperties();
        process(resultItems, task, properties);
    }

    protected void process(ResultItems resultItems, Task task, ProcessorProperties properties) {

    }

    @Override
    public void process(Page page) {
        ProcessorProperties properties = getProcessProperties();
        this.process(page, properties);
    }

    //主函数，执行爬虫
    @SneakyThrows
    public void start() {
        ProcessorProperties properties = getProcessProperties();
        // todo 网页内容去重，simhash

        // todo url 去重 重复的一定找到，不重复的会误判
//        DuplicateRemovedScheduler scheduler = new QueueScheduler()
//                .setDuplicateRemover(new BloomFilterDuplicateRemover(1000000));


        // creaeteDownloader
        Downloader downloader = creaeteDownloader(properties);

        // createRequest
        Request request = createRequest(properties);

        request.setDownloader(downloader);

        ExecutorService threadPool = Executors.newFixedThreadPool(1);
        Spider spider = Spider.create(this)
                .addRequest(request).addPipeline(this)
                .setSpiderListeners(Collections.singletonList(new DingDingSpiderListener(properties)))
                .thread(threadPool, 1)
                .setDownloader(downloader);
        //.setScheduler(scheduler);
        spider.run();//执行

    }

    public abstract Request createRequest(ProcessorProperties properties);


    protected static void setHeader(Request request, String header, String value) {
        if (StringUtils.isNoneBlank(value)) {
            if (Objects.equals(header,"Cookie")){
                for (String val : value.split(";")) {
                    String[] split = val.split("=");
                    if (split.length==2) {
                        request.addCookie(split[0].trim(),split[1].trim());
                    }
                }
                return;
            }
            request.addHeader(header, value);
        }

    }

    protected static void setHeader(Site site, String header, String value) {

        if (StringUtils.isNoneBlank(value)) {
            if (Objects.equals(header,"Cookie")){
                for (String val : value.split(";")) {
                    String[] split = val.split("=");
                    if (split.length==2) {
                        site.addCookie(split[0].trim(),split[1].trim());
                    }
                }
                return;
            }
            site.addHeader(header, value);
        }
    }


//    private Downloader creaeteDownloader(PageProcessProperties properties) {
//        // get proxyprovider
//        Proxy proxy = new Proxy("118.178.239.78", 80,"http");
//        CustomizeProxyProvider proxyProvider = new CustomizeProxyProvider();
//        proxyProvider.addProxy(proxy);
//        String downloaderStr = properties.getDownloader();
//        Downloader downloader;
//        if ("chrome".equals(downloaderStr)){
//            downloader = new SingleChromeDownloader(properties);
//        }else {
//            HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
//            // httpClientDownloader.setProxyProvider(proxyProvider);
//            downloader = httpClientDownloader;
//        }
//        return downloader;
//    }

    public abstract Downloader creaeteDownloader(ProcessorProperties properties);

    private ProcessorProperties getProcessProperties() {
        WebMagicProperties properties = SpringUtil.getBean(WebMagicProperties.class);
        String[] beanNamesForType = SpringUtil.getBeanNamesForType(this.getClass());
        return properties.getProcess().get(beanNamesForType[0]);
    }

    public final Site getSite() {
        ProcessorProperties properties = getProcessProperties();
        Site site = Site.me();
        site.setCharset(properties.getCharset());
        site.setUserAgent(properties.getUserAgent());
        // 测试使用，实际没用
        setHeader(site, "Cookie", properties.getCookie());

        // download成功后停顿时间，和失败且CycleRetryTimes==0的停顿时间
        site.setSleepTime(1000);

        // 失败后的重试次数
        site.setCycleRetryTimes(properties.getCycleRetryTimes());
        // 失败后的重试的间隔时间
        site.setRetrySleepTime(properties.getRetrySleepTime());

        // http请求重试次数
        site.setRetryTimes(0);
        // http请求sockettime
        site.setTimeOut(30000);

        return site;
    }


}
